package v2;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;

import pipeline.JsonFilePipeline;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
import util.BaseUtil;
import vo.PinPaiVO;

public class JdMain {
	
	private static final Logger log = Logger.getLogger(JdMain.class);
	
	public static void main(String[] args) throws Exception {
		long startTime = new Date().getTime();
		if (args.length < 3){
			System.out.println("<inputFile><encode><outputFilePath>");
			return;
		}
		
		String inputFilePath = args[0];
		String encode = args[1];
		List<String> needPinPais = BaseUtil.readFileText(inputFilePath, encode);
		
		//1、获取所有手机品牌的入口url
		JdPinPaiUrlListPageProcessor jdPinPaiUrlListPageProcessor = new JdPinPaiUrlListPageProcessor();
		Map<String, PinPaiVO> jdPinPaiMap = jdPinPaiUrlListPageProcessor.getPinPaiMap();
		
		List<String> userAgents = new ArrayList<>();
		userAgents.add("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36");
		userAgents.add("Mozilla/5.0 (Windows NT 6.3; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0");
		userAgents.add("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C; InfoPath.3; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729");
		userAgents.add("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)");
		userAgents.add("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)");
		userAgents.add("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)");
		userAgents.add("Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
		
		if(jdPinPaiMap != null){
			String outputFilePath = args[2];
			for(Map.Entry<String, PinPaiVO> entry : jdPinPaiMap.entrySet()){
				String name = entry.getValue().getName();
				String pinpaiUrl = entry.getValue().getUrl(); 
				//2、判断是否是要抓取的品牌，是则继续
				String pinPai = BaseUtil.toCrawl(name, needPinPais);
				if("".equals(pinPai)){
					continue;
				}
				//3、输出手机信息到相应文件中
				String fileName = new String(pinPai.getBytes(),"UTF-8")+".json";
				
				JdPageProcessor jdPageProcessor = new JdPageProcessor();
				jdPageProcessor.setEntryUrl(pinpaiUrl);
				jdPageProcessor.setUserAgent(userAgents.get((int)(Math.random()*userAgents.size())));
				Spider spider = Spider.create(jdPageProcessor).scheduler(new PriorityScheduler())
					.addPipeline(new JsonFilePipeline(outputFilePath, fileName));
				spider.addUrl(pinpaiUrl);
				spider.thread(5).run();
			}
		}
		long endTime = new Date().getTime();
		log.info("京东执行时间："+String.valueOf((endTime-startTime)));
	}
}
